from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
from os import walk

ifn = "/home/weijiang/workspace/ORIGINAL_POLY_IRTK/index_remapped.lex_32Bit"
ofn = "/home/weijiang/workspace/ORIGINAL_POLY_IRTK/index_remapped.lex"

ifh = open(ifn,"rb")
ofh = open(ofn,"wb")

print "Reading",ifn
print "Writing",ofn

#step2:
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = 0
termCounter = 0
while numOfBytesRead < fileSize:
    
    byteString_numOfLayers = ifh.read(4)
    byteString_termLen = ifh.read(4)
    
    (numOfLayers,) = unpack( "i", byteString_numOfLayers)
    (termLen,) = unpack( "i", byteString_termLen)
    
    byteString_term = ifh.read(termLen)
    
    byteString_numOfDocs = ifh.read(4)
    (numOfDocs,) = unpack( "i", byteString_numOfDocs)
    
    byteString_numOfChunks = ifh.read(4)
    (numOfChunks,) = unpack( "i", byteString_numOfChunks)
    
    byteString_numOfChunksLastBlock = ifh.read(4)
    (numOfChunksLastBlock,) = unpack( "i", byteString_numOfChunksLastBlock)
    
    byteString_numOfBlocks = ifh.read(4)
    (numOfBlocks,) = unpack( "i", byteString_numOfBlocks)
    
    byteString_blockNum = ifh.read(4)
    (blockNum,) = unpack( "i", byteString_blockNum)
    
    byteString_chunkNum = ifh.read(4)
    (chunkNum,) = unpack( "i", byteString_chunkNum)
    
    byteString_scoreThreshold = ifh.read(4)
    (scoreThreshold,) = unpack( "f", byteString_scoreThreshold)
    
    byteString_externalIndexOffset_OLD = ifh.read(4) # for the externalIndexOffset 
    (externalIndexOffset_OLD,) = unpack( "I", byteString_externalIndexOffset_OLD)
    numOfBytesRead += 4*2 + termLen + 4*8
    print termCounter,byteString_term,numOfDocs,numOfBytesRead,externalIndexOffset_OLD
    termCounter += 1
    
    
    ofh.write(byteString_numOfLayers)
    ofh.write(byteString_termLen)
    ofh.write(byteString_term)
    ofh.write(byteString_numOfDocs)
    ofh.write(byteString_numOfChunks)
    ofh.write(byteString_numOfChunksLastBlock)
    ofh.write(byteString_numOfBlocks)
    ofh.write(byteString_blockNum)
    ofh.write(byteString_chunkNum)
    ofh.write(byteString_scoreThreshold)
    ofh.write(pack("2I",0,0))

print "Overall:"
print "ifn:",ifn
print "ofn:",ofn
print "# of terms in the lexicon:",termCounter
print "DONE."
print
ifh.close()
ofh.close()
